#import operator
import sys


def dealWithFile1():
    print "dealWithFile1*******************"
    return_query_Dict = {}
    inputFileName1 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/query_features_2011_adhoc_sample_head_8.txt"
    inputFileHandler1 = open(inputFileName1, "r")
    #keyWordsDict = {}
    
    for currentLine in inputFileHandler1.readlines():
        currentLineElements = currentLine.split("\t")
        queryNumber = currentLineElements[0].strip()
        adhocTopicNumber = currentLineElements[2].strip()
        relevenceScore = currentLineElements[4].strip()
        queryInfoString = queryNumber + " " + adhocTopicNumber + " " + relevenceScore + " "
        
        queryContent = currentLineElements[1].strip()
        warcTrecID = currentLineElements[3].strip()
        
        returnTurple = (queryContent, warcTrecID)
        if returnTurple not in return_query_Dict:
            return_query_Dict[returnTurple] = queryInfoString
        else:
            print "Error1"
            print returnTurple
            sys.exit(1)
            
    
    return return_query_Dict
    #The key is the thing we hunt for.
      
    '''   
        for currentTerm in queryContentElement:
            key = currentTerm + "_" + warcTrecID
            if key not in keyWordsDict:
                keyWordsDict[key] = 1
            else:
                keyWordsDict[key] += 1
    
    keyWordsDict
    sorted_keyWordsDict = sorted(keyWordsDict.iteritems(), key=operator.itemgetter(0))
    
    print len(keyWordsDict)
    print type(sorted_keyWordsDict)
    
    
    for currentElement in sorted_keyWordsDict:
        (queryTerm_warcID,freq) = currentElement
        print queryTerm_warcID
        #print "freq:",freq
    '''
    inputFileHandler1.close()

def dealWithFile2( queryDict ):
    print "dealWithFile2*******************"
    #print len(queryDict)
    #print queryDict
    
    documentDict = {}
    
    inputFileName2 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/document_features.txt"
    inputFileHandler2 = open(inputFileName2, "r")
    for currentLine in inputFileHandler2.readlines():
        currentLineElements = currentLine.strip().split(" ")
        url = currentLineElements[1]
        hamProbability = currentLineElements[3]
        isPopular = currentLineElements[4]
        domainName = currentLineElements[5]
        urlLength = currentLineElements[6]
        documentSizeInBytes = currentLineElements[7]
        urlNestingLevel = currentLineElements[8]
        numberOfWords = currentLineElements[9]
        numberOfDistantWords = currentLineElements[10]
        
        documentInfoString = url + " " + hamProbability + " " + isPopular + " " + domainName + " " + urlLength + " " + documentSizeInBytes + " " + urlNestingLevel + " " + numberOfWords + " " + numberOfDistantWords + " "; 
        
        warcTrecID = currentLineElements[0]
        docID = currentLineElements[2]
        if (warcTrecID,docID) not in documentDict:
            documentDict[ (warcTrecID,docID) ] = documentInfoString
        else:
            print "Error1"
            sys.exit(1)
            
    inputFileHandler2.close()
    #print len(documentDict)
    print documentDict
    
    #Let's construct a query-document dict
    return_query_document_Dict = {}
    
    for currentElement in queryDict:
        (queryContent,warcTrecID) = currentElement
        
        for eachWARCIDDocIDElement in documentDict:
            (currentWARCID,currentDocID) = eachWARCIDDocIDElement
            
            if warcTrecID == currentWARCID:
                newTurple = (queryContent,currentDocID)
                
                if newTurple not in return_query_document_Dict:
                    return_query_document_Dict[newTurple] = queryDict[currentElement] + documentDict[ (warcTrecID,currentDocID) ]
                else:
                    print "Error2"
                    print newTurple
                    sys.exit(1)

    #print len(return_query_document_Dict)
    #print return_query_document_Dict
    return return_query_document_Dict

def dealWithFile3( query_document_Dict ):
    print "dealWithFile3*******************"
    #print len(query_document_Dict)
    #print query_document_Dict
    
    
    
    termDict = {}
    
    inputFileName3 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/term_features.txt"
    inputFileHandler3 = open(inputFileName3, "r")
    for currentLine in inputFileHandler3.readlines():
        currentLineElements = currentLine.strip().split(" ")
        word = currentLineElements[0]
        docID = currentLineElements[1]
        position = currentLineElements[2]
        context = currentLineElements[3]
        BM25Score = currentLineElements[4]
        key = (word, docID, BM25Score)
        if key not in termDict:
            termDict[key] = [(position,context)]
        else:
            #print termDict[key]
            termDict[key] = termDict[key] + [(position,context)]
            
    inputFileHandler3.close()
    #print len(termDict)
    
    
    #for key in termDict:
    #    print key,termDict[key]
    #print termDict[("day","55")]
    #print termDict[("day","7")]
    
    
    #Let's construct a query_document_term dict
    return_query_document_term_Dict = {}
    
    noPositionCounter = 0
    for currentElement in query_document_Dict:
        (queryContent,docID) = currentElement
        queryContentElements = queryContent.split(" ")
        
        for queryTerm in queryContentElements:
            
            for eachTuple in termDict:
                (word,wordDocID,wordBM25) = eachTuple
                if word == queryTerm and wordDocID = docID:
                    #this is the fucking part.
            
            
            if (queryTerm, docID) in termDict:
                newTurpleString = "(" + queryTerm + "," + docID + "," + str( len(termDict[(queryTerm, docID)]) ) + ","
                for eachPositionInfo in termDict[(queryTerm, docID)][: len(termDict[(queryTerm, docID)])-1]:
                    newTurpleString += eachPositionInfo + ","
                newTurpleString += termDict[(queryTerm, docID)][-1]
                newTurpleString += ")"
                #print newTurpleString
                #newTurple = tuple(newTurpleString)
                
                if newTurpleString not in return_query_document_term_Dict:
                    return_query_document_term_Dict[newTurpleString] = query_document_Dict[currentElement]
                
            else:
                #print "NO Key:",queryTerm, docID
                noPositionCounter += 1
                newTurpleString = "(" + queryTerm + "," + docID + "," + str( 0 ) + ")"
                #newTurple = tuple(newTurpleString)
                if newTurpleString not in return_query_document_term_Dict:
                    return_query_document_term_Dict[newTurpleString] = query_document_Dict[currentElement]
                else:
                    print "Error3"
                    print newTurpleString
                    sys.exit(1)
                
                
    
    print "noPositionCounter:",noPositionCounter
    
    return return_query_document_term_Dict
    

    

#main program
dict1 = dealWithFile1()
print len(dict1)
for queryWARCTRECIDTupleElement in dict1:
    print queryWARCTRECIDTupleElement,dict1[queryWARCTRECIDTupleElement]

dict2 = dealWithFile2( dict1 )
print len(dict2)
for queryWARCTRECIDDocIDTupleElement in dict2:
    print queryWARCTRECIDDocIDTupleElement,dict2[queryWARCTRECIDDocIDTupleElement]


dict3 = dealWithFile3( dict2 )
'''
print len(dict3)
for queryDocIDTermTupleElement in dict3:
    print queryDocIDTermTupleElement
'''
#print len(dict3)
#for currentElement in dict3:
#    print currentElement,dict3[currentElement]
 
    
    
    
